{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Matrix.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN9MHnewXIidW2S/4YClk2m",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tomasonjo/blogs/blob/master/matrix/MatrixNLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "* Updated to GDS 2.0 version\n",
        "* Link to original blog post: https://towardsdatascience.com/construct-the-matrix-interaction-network-based-on-the-movie-script-738b4fa9b46d"
      ],
      "metadata": {
        "id": "5X_tdaUHUHWd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xU9656o614cy",
        "outputId": "fbc284b1-3d87-400b-a1a7-f4e08231fe57"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "tesseract-ocr is already the newest version (4.00~git2288-10f4998a-2).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 69 not upgraded.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "poppler-utils is already the newest version (0.62.0-2ubuntu2.12).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 69 not upgraded.\n",
            "Requirement already satisfied: selenium in /usr/local/lib/python3.7/dist-packages (4.1.0)\n",
            "Requirement already satisfied: neo4j in /usr/local/lib/python3.7/dist-packages (4.4.0)\n",
            "Requirement already satisfied: pytesseract in /usr/local/lib/python3.7/dist-packages (0.3.8)\n",
            "Requirement already satisfied: pdf2image in /usr/local/lib/python3.7/dist-packages (1.16.0)\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.7/dist-packages (3.2.1)\n",
            "Requirement already satisfied: trio~=0.17 in /usr/local/lib/python3.7/dist-packages (from selenium) (0.19.0)\n",
            "Requirement already satisfied: trio-websocket~=0.9 in /usr/local/lib/python3.7/dist-packages (from selenium) (0.9.2)\n",
            "Requirement already satisfied: urllib3[secure]~=1.26 in /usr/local/lib/python3.7/dist-packages (from selenium) (1.26.7)\n",
            "Requirement already satisfied: outcome in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (1.1.0)\n",
            "Requirement already satisfied: idna in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (2.10)\n",
            "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (21.2.0)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (1.2.0)\n",
            "Requirement already satisfied: async-generator>=1.9 in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (1.10)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.7/dist-packages (from trio~=0.17->selenium) (2.4.0)\n",
            "Requirement already satisfied: wsproto>=0.14 in /usr/local/lib/python3.7/dist-packages (from trio-websocket~=0.9->selenium) (1.0.0)\n",
            "Requirement already satisfied: pyOpenSSL>=0.14 in /usr/local/lib/python3.7/dist-packages (from urllib3[secure]~=1.26->selenium) (21.0.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from urllib3[secure]~=1.26->selenium) (2021.10.8)\n",
            "Requirement already satisfied: cryptography>=1.3.4 in /usr/local/lib/python3.7/dist-packages (from urllib3[secure]~=1.26->selenium) (36.0.1)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.7/dist-packages (from cryptography>=1.3.4->urllib3[secure]~=1.26->selenium) (1.15.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.12->cryptography>=1.3.4->urllib3[secure]~=1.26->selenium) (2.21)\n",
            "Requirement already satisfied: six>=1.5.2 in /usr/local/lib/python3.7/dist-packages (from pyOpenSSL>=0.14->urllib3[secure]~=1.26->selenium) (1.15.0)\n",
            "Requirement already satisfied: h11<1,>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.12.0)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from neo4j) (2018.9)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from pytesseract) (7.1.2)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.8.2)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.6.1)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.3.0)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.4.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.6)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.6)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (21.3)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.19.5)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (0.4.1)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /usr/local/lib/python3.7/dist-packages (from spacy) (8.0.13)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.8)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (4.62.3)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.10.0.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.26.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.8.2)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy) (3.0.6)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy) (1.0.1)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.0.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy) (57.4.0)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.4.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy) (2.11.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy) (3.0.6)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy) (5.2.1)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.0.8)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy) (2.0.1)\n",
            "Collecting en-core-web-sm==3.2.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0-py3-none-any.whl (13.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 13.9 MB 5.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.3.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-sm==3.2.0) (3.2.1)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (8.0.13)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.6.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.6)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.11.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (21.3)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.19.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (57.4.0)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.4.0)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.4.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.8.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.3.0)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.1)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.4.2)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.8.2)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.10.0.2)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (4.62.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.26.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.6)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /usr/local/lib/python3.7/dist-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.6)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (5.2.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2021.10.8)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.8)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.26.7)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.1)\n",
            "Installing collected packages: en-core-web-sm\n",
            "  Attempting uninstall: en-core-web-sm\n",
            "    Found existing installation: en-core-web-sm 2.2.5\n",
            "    Uninstalling en-core-web-sm-2.2.5:\n",
            "      Successfully uninstalled en-core-web-sm-2.2.5\n",
            "Successfully installed en-core-web-sm-3.2.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n"
          ]
        }
      ],
      "source": [
        "!sudo apt install tesseract-ocr\n",
        "!sudo apt-get install poppler-utils \n",
        "# Install dependencies\n",
        "!pip install -U selenium neo4j pytesseract pdf2image spacy --upgrade\n",
        "!python -m spacy download en_core_web_sm"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup selenium\n",
        "!apt-get update\n",
        "!apt install chromium-chromedriver\n",
        "!cp /usr/lib/chromium-browser/chromedriver /usr/bin\n",
        "import sys\n",
        "sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')\n",
        "from selenium import webdriver\n",
        "chrome_options = webdriver.ChromeOptions()\n",
        "chrome_options.add_argument('--headless')\n",
        "chrome_options.add_argument('--no-sandbox')\n",
        "chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "wd = webdriver.Chrome('chromedriver',chrome_options=chrome_options)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QzliV2US4yQR",
        "outputId": "bcc0c554-b0f1-43ac-d81f-ba7aba2a0932"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Hit:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "Hit:2 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:3 http://security.ubuntu.com/ubuntu bionic-security InRelease\n",
            "Hit:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "Ign:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:7 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n",
            "Hit:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:9 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Hit:10 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:11 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n",
            "Hit:12 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Hit:13 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Reading package lists... Done\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "chromium-chromedriver is already the newest version (95.0.4638.69-0ubuntu0.18.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 69 not upgraded.\n",
            "cp: '/usr/lib/chromium-browser/chromedriver' and '/usr/bin/chromedriver' are the same file\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:12: DeprecationWarning: use options instead of chrome_options\n",
            "  if sys.path[0] == '':\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Construct the Matrix interaction network based on the movie script\n",
        "## Combine web scraping, OCR, and entity recognition to construct and analyze the Matrix interaction network in Neo4j\n",
        "Christmas is just around the corner, and with it, comes the newest Matrix movie. I can't think of a better way to wait till the movie is released than to perform a network analysis of the first Matrix movie.\n",
        "## Agenda\n",
        "This blog post will present how to combine web scraping, OCR, and NLP techniques to construct the Matrix interaction network. \n",
        "* Scraping Matrix fandom page with Selenium\n",
        "* Using PyTesseract to read the Matrix movie script PDF\n",
        "* Extract characters in each scene by using the SpaCy's rule-based matcher\n",
        "* Construct and analyze the character's co-occurrence network in Neo4j\n",
        "\n",
        "I have already performed a similar analysis based on the Harry Potter book, and this time we will be using the Matrix movie script.\n",
        "\n",
        "## Scraping Matrix Fandom page with Selenium\n",
        "We will begin by scraping the Matrix Fandom page to get the list of characters that appeared in the movie. As mentioned, we will be using the Selenium library to achieve this. The content of the fandom page is available under the CC BY 4.0 license.\n",
        "In the first step, we will extract the names and the links of characters that appeared in the first Matrix movie."
      ],
      "metadata": {
        "id": "M2SU_waapf7m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "wikifan_url = \"https://matrix.fandom.com/wiki/Category:Characters_in_The_Matrix\"\n",
        "\n",
        "member_list = []\n",
        "wd.get(wikifan_url)\n",
        "members = wd.find_elements_by_class_name(\"category-page__member-link\")\n",
        "for m in members:\n",
        "  member_list.append({'url':m.get_attribute('href'), 'name': m.text})\n",
        "\n",
        "# Manually append Trinity\n",
        "member_list.append({'url': 'https://matrix.fandom.com/wiki/Trinity', 'name': 'Trinity'})"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kzv8cJ8w38WZ",
        "outputId": "c525088f-54d3-46ea-fba5-93581c308e81"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Just because we can, we will also extract detailed information from personal pages of characters."
      ],
      "metadata": {
        "id": "sXpbTGxwpvvy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for m in member_list:\n",
        "  wd.get(m['url'])\n",
        "  elements = wd.find_elements_by_class_name(\"pi-data\")\n",
        "  for e in elements:\n",
        "    try:\n",
        "      label = e.find_element_by_tag_name(\"h3\")\n",
        "      value = e.find_element_by_tag_name(\"div\")\n",
        "      m[label.text] = value.text\n",
        "    except:\n",
        "      pass\n",
        "\n",
        "  "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i8kMbdUB5PHV",
        "outputId": "c60738e4-602a-4c38-849b-e8fc4befd81a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: DeprecationWarning: find_elements_by_* commands are deprecated. Please use find_elements() instead\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n",
            "/usr/local/lib/python3.7/dist-packages/selenium/webdriver/remote/webelement.py:341: UserWarning: find_element_by_* commands are deprecated. Please use find_element() instead\n",
            "  warnings.warn(\"find_element_by_* commands are deprecated. Please use find_element() instead\")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Before continuing, we will store the character information into Neo4j. If you are using the Colab notebook, then it would be easiest to create either a free Neo4j Sandbox or free Aura database instance to store the results.\n",
        "Once you have created the Sandbox or the Aura environment, simply copy the connection details into the notebook."
      ],
      "metadata": {
        "id": "Cb24DLv8p0DD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from neo4j import GraphDatabase\n",
        "# Change the host and user/password combination to your neo4j\n",
        "# Will not work with a localhost bolt url\n",
        "host = 'bolt://44.200.249.124:7687'\n",
        "user = 'neo4j'\n",
        "password = 'battle-manpower-sand'\n",
        "driver = GraphDatabase.driver(host,auth=(user, password))"
      ],
      "metadata": {
        "id": "s2taZG9V3xiH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now that you have defined the connection to your Neo4j instance, you can go ahead and import the characters' information."
      ],
      "metadata": {
        "id": "cRWs1Q0Gp4BB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "entity_query = \"\"\"\n",
        "UNWIND $data as row\n",
        "CREATE (c:Character)\n",
        "SET c += row\n",
        "\"\"\"\n",
        "with driver.session() as session:\n",
        "  session.run(entity_query, {'data': member_list})"
      ],
      "metadata": {
        "id": "Gm8jt3y333nW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "At this moment, there are no connections in the database, just lonely and isolated nodes. If you wanted to, you could refactor some of the node properties such as Spouse to a relationship. However, we will skip this part and move on to constructing a co-occurrence network based on the movie script.\n",
        "## Using PyTesseract to read the Matrix movie script PDF\n",
        "The movie script is available on the Daily Script web page in PDF format. While no explicit license is stated, the web page says that the scripts are available for educational purposes, so we are good to go.\n",
        "We will use PyTesseract library to transform the PDF into a text format."
      ],
      "metadata": {
        "id": "oH0BUeAJp7u6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "import pdf2image\n",
        "import pytesseract\n",
        "\n",
        "pdf_link = \"https://www.dailyscript.com/scripts/the_matrix.pdf\"\n",
        "\n",
        "pdf = requests.get(pdf_link)\n",
        "doc = pdf2image.convert_from_bytes(pdf.content)\n",
        "\n",
        "# Get the article text\n",
        "article = []\n",
        "for page_number, page_data in enumerate(doc):\n",
        "    # First page is the title\n",
        "    if page_number == 0:\n",
        "      continue\n",
        "    txt = pytesseract.image_to_string(page_data, lang='eng').encode(\"utf-8\")\n",
        "    article.append(txt.decode(\"utf-8\"))\n",
        "article_txt = \" \".join(article)"
      ],
      "metadata": {
        "id": "XcF4eb3u2qE_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This process takes around 15 minutes, so you can use this time to take a break and perhaps stretch your legs.\n",
        "Before moving on to the character extraction step, we will perform a simple text cleanup and split the script by scenes."
      ],
      "metadata": {
        "id": "rYZbAALTqC2E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# a bit of cleaning\n",
        "article_clean_txt = \"\\n\".join([line for line in article_txt.split(\"\\n\") if not \"THE MATRIX\" in line and not \"CONTINUED\" in line])"
      ],
      "metadata": {
        "id": "eEhFVZh5EMSm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Optionally store to file\n",
        "with open('/matrix_script.txt', 'w') as writefile:\n",
        "    writefile.write(article_clean_txt)\n",
        "\n",
        "from google.colab import files\n",
        "files.download('/matrix_script.txt')"
      ],
      "metadata": {
        "id": "irDkqb4u2WBM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "outputId": "8de54049-476f-43ef-9717-c651ad347725"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_c4ea18cc-50b2-4ce3-bd1e-41592b782403\", \"matrix_script.txt\", 136738)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Split by scenes\n",
        "scenes = []\n",
        "single_scene = []\n",
        "for line in article_clean_txt.split(\"\\n\"):\n",
        "  # If empty line\n",
        "  if not line or line.startswith(\"OMITTED\"):\n",
        "    continue\n",
        "  if line.startswith(\"INT.\") or line.startswith(\"EXT.\") or line.startswith(\"THE END\"):\n",
        "    scenes.append((\"\\n\").join(single_scene))\n",
        "    single_scene = []\n",
        "  single_scene.append(line)\n",
        "\n",
        "scene_names = [el.split(\"\\n\")[0] for el in scenes]"
      ],
      "metadata": {
        "id": "0hjOCB1XMXsZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now that we have preprocessed the text, we will go ahead and identify all the characters that appear in a particular scene. Since we already know which characters to expect from our fandom scraping process, we'll use SpaCy's rule-based matcher to identify characters.\n",
        "A character name can appear in two forms. First, if the character is talking, its name is uppercased in the text. The second form is the title-cased version, where a character is mentioned by other persons or mentioned in the scene description. SpaCy makes it really easy to describe these two patterns.\n",
        "We will also omit the The word from the pattern definition. For example, the fandom page contains the character The Oracle. Therefore, we will skip the word The and only search for Oracle pattern.\n",
        "The following code will construct the SpaCy's matcher object used in the next step to identify characters."
      ],
      "metadata": {
        "id": "tZXIX9DqqGh6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_matcher_patterns(name):\n",
        "  \"\"\"Function that construct a SpaCy rule-based pattern from a name\"\"\"\n",
        "  matcher_pattern = []\n",
        "  clean_name = name.replace(\"The\", \"\").strip()\n",
        "  parts_of_name = clean_name.split(\" \")\n",
        "  # Append the capitalized version\n",
        "  matcher_pattern.append([{\"LOWER\": n.lower(), \"IS_TITLE\": True} for n in parts_of_name])\n",
        "  # Append the uppercased version\n",
        "  matcher_pattern.append([{\"LOWER\": n.lower(), \"IS_UPPER\": True} for n in parts_of_name])\n",
        "  return matcher_pattern"
      ],
      "metadata": {
        "id": "oUJNww-irKbb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "from spacy.matcher import Matcher\n",
        "\n",
        "nlp = spacy.load('en_core_web_sm')\n",
        "matcher = Matcher(nlp.vocab)\n",
        "\n",
        "for m in member_list:\n",
        "  matcher.add(m['name'], get_matcher_patterns(m['name']))"
      ],
      "metadata": {
        "id": "mmh8emaIuqaL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have the text and the entity matcher ready. We will iterate over scenes, identify all the characters that appear in it, and store the results directly into Neo4j in one step."
      ],
      "metadata": {
        "id": "_27DMM1sqLeu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "x = 0\n",
        "\n",
        "for s, sn in zip(scenes, scene_names):\n",
        "  characters = set()\n",
        "  doc = nlp(s.replace(\"\\n\", \" \"))\n",
        "  matches = matcher(doc)\n",
        "  \n",
        "  for match_id, start, end in matches:\n",
        "    characters.add(str(doc[start:end]).lower())\n",
        "  \n",
        "  entity_query = \"\"\"\n",
        "      MERGE (s:Scene {id: $scene_id})\n",
        "      SET s.title = $scene_title\n",
        "      WITH s\n",
        "      UNWIND $characters as char\n",
        "      MATCH (c:Character)\n",
        "      WHERE toLower(c.name) = CASE WHEN NOT char IN [\"oracle\", \"priestess\"] THEN char ELSE \"the \" + char END\n",
        "      MERGE (c)-[:IN_SCENE]->(s)\n",
        "      \"\"\"\n",
        "  with driver.session() as session:\n",
        "    session.run(entity_query, {'scene_id': x, 'scene_title': sn, 'characters': list(characters)})\n",
        "\n",
        "  x += 1\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "e8HKDdY_u7xc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "That's how easy it is to extract information from the movie script and store the output into Neo4j.\n",
        "Now we will move on to the network analysis part of this blog. First, we will evaluate all the characters that didn't appear in any scene."
      ],
      "metadata": {
        "id": "irqmAuCEqRRr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "def read_query(query, params=None):\n",
        "    with driver.session() as session:\n",
        "        result = session.run(query, params)\n",
        "        return pd.DataFrame([r.values() for r in result], columns=result.keys())"
      ],
      "metadata": {
        "id": "ua3ndpdIwWJK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "read_query(\"\"\"\n",
        "MATCH (n:Character) WHERE NOT (n)--()\n",
        "RETURN n.name\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "kT93Lk-l7Ep7",
        "outputId": "3271e751-75be-4e02-8d52-f282f93fa66f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 426
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>n.name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Austin</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Crawford</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>FedEx Man</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Garcia</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Green</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Kim</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Law Enforcement</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Nguyen</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>S.W.A.T.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Security Guards</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>System aligned bluepills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Woman in Red</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                      n.name\n",
              "0                     Austin\n",
              "1                   Crawford\n",
              "2                  FedEx Man\n",
              "3                     Garcia\n",
              "4                      Green\n",
              "5                        Kim\n",
              "6            Law Enforcement\n",
              "7                     Nguyen\n",
              "8                   S.W.A.T.\n",
              "9            Security Guards\n",
              "10  System aligned bluepills\n",
              "11              Woman in Red"
            ]
          },
          "metadata": {},
          "execution_count": 92
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Twelve characters weren't identified in any scene. I don't remember any of the characters except the Woman in Red. Probably she is mentioned as the red woman or something similar in the text, so our pattern matcher didn't identify her. Of course, we could fine-tune the patterns to include these types of exceptions, but overall, it seems that the main characters were identified.\n",
        "Next, we will examine the characters that appeared in most scenes."
      ],
      "metadata": {
        "id": "NDE0p1BcqV6o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "read_query(\"\"\"\n",
        "MATCH (n:Character)\n",
        "RETURN n.name AS character,\n",
        "       size((n)-[:IN_SCENE]->()) as scenes\n",
        "ORDER BY scenes DESC\n",
        "LIMIT 5\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "798wpGJO7KkA",
        "outputId": "78e644ce-7d55-4a47-932a-e7f104c3cf9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>character</th>\n",
              "      <th>scenes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Neo</td>\n",
              "      <td>115</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Morpheus</td>\n",
              "      <td>87</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Trinity</td>\n",
              "      <td>77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Tank</td>\n",
              "      <td>58</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Agent Smith</td>\n",
              "      <td>36</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     character  scenes\n",
              "0          Neo     115\n",
              "1     Morpheus      87\n",
              "2      Trinity      77\n",
              "3         Tank      58\n",
              "4  Agent Smith      36"
            ]
          },
          "metadata": {},
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Nothing shocking here. Neo appeared in more than half of the scenes, followed by Morpheus and Trinity. Only Emil Eifrem is surprisingly missing from this list.\n",
        "We can define the co-occurrence event as a pair of characters that appeared in the same scene. In this context, co-occurrence can also be understood as an interaction. The more scenes a duo of characters appeared in, the more they interacted in the movie.\n",
        "We can evaluate which characters interacted the most by executing the following Cypher statement."
      ],
      "metadata": {
        "id": "WYXMPYxkqael"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "read_query(\"\"\"\n",
        "MATCH (n1:Character)-[:IN_SCENE]->()<-[:IN_SCENE]-(n2:Character)\n",
        "WHERE id(n1) < id(n2)\n",
        "RETURN n1.name AS character1,\n",
        "       n2.name AS character2,\n",
        "       count(*) AS count\n",
        "ORDER BY count DESC\n",
        "LIMIT 5\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "CUsjI8c4-JaJ",
        "outputId": "8f9c53d7-dd96-458b-c47a-6e151c2125aa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>character1</th>\n",
              "      <th>character2</th>\n",
              "      <th>count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Morpheus</td>\n",
              "      <td>Neo</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Neo</td>\n",
              "      <td>Trinity</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Morpheus</td>\n",
              "      <td>Trinity</td>\n",
              "      <td>34</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Neo</td>\n",
              "      <td>Tank</td>\n",
              "      <td>26</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Morpheus</td>\n",
              "      <td>Tank</td>\n",
              "      <td>26</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  character1 character2  count\n",
              "0   Morpheus        Neo     59\n",
              "1        Neo    Trinity     55\n",
              "2   Morpheus    Trinity     34\n",
              "3        Neo       Tank     26\n",
              "4   Morpheus       Tank     26"
            ]
          },
          "metadata": {},
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Most interactions occurred between Neo, Morpheus, Trinity, and Tank. If you watched the movie, this all makes sense.\n",
        "Lastly, we can infer the co-occurrence network between characters and perform a network analysis of it. We will simply count the number of interactions between a pair of characters and store the information as a relationship."
      ],
      "metadata": {
        "id": "WiRl2o2NqeqY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "read_query(\"\"\"\n",
        "MATCH (n1:Character)-[:IN_SCENE]->()<-[:IN_SCENE]-(n2:Character)\n",
        "WHERE id(n1) < id(n2)\n",
        "WITH n1, n2, count(*) AS count\n",
        "MERGE (n1)-[r:INTERACTS]-(n2)\n",
        "SET r.weight = count\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "e6eQMgM7-bQm",
        "outputId": "3fbe2655-01c8-4a93-aba2-a15ad0d6f66c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 32
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: []\n",
              "Index: []"
            ]
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Graph Data Science library\n",
        "Neo4j features a Graph Data Science library with more than 50 graph algorithms ranging from centrality, community detection, and node embedding categories.\n",
        "We will use PageRank to evaluate node importance and Louvain to determine the community structure of the inferred co-occurrence network. Instead of inspecting each algorithm result separately, we will store the results and construct a network visualization that visualizes both node importance and community structure.\n",
        "First, we have to project an in-memory graph in order to be able to execute graph algorithms on it. Notice that we project the co-occurrence relationship as undirected. For example, if Neo interacted with Trinity, this directly implies that Trinity also interacted with Neo."
      ],
      "metadata": {
        "id": "NH-ZYRPgqjcR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "read_query(\"\"\"\n",
        "CALL gds.graph.project(\"matrix\", \"Character\", {INTERACTS: {orientation:\"UNDIRECTED\", properties:\"weight\"}})\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "9iwdgR5k_CcI",
        "outputId": "04145981-1ef1-4350-de27-897603147ab0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>nodeProjection</th>\n",
              "      <th>relationshipProjection</th>\n",
              "      <th>graphName</th>\n",
              "      <th>nodeCount</th>\n",
              "      <th>relationshipCount</th>\n",
              "      <th>createMillis</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>{'Character': {'properties': {}, 'label': 'Cha...</td>\n",
              "      <td>{'INTERACTS': {'orientation': 'UNDIRECTED', 'a...</td>\n",
              "      <td>matrix</td>\n",
              "      <td>33</td>\n",
              "      <td>176</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      nodeProjection  ... createMillis\n",
              "0  {'Character': {'properties': {}, 'label': 'Cha...  ...           55\n",
              "\n",
              "[1 rows x 6 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we can go ahead and execute the weighted PageRank algorithm and store the results back to Neo4j."
      ],
      "metadata": {
        "id": "a27Y1pmzqnTa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "read_query(\"\"\"\n",
        "CALL gds.pageRank.write(\"matrix\", {relationshipWeightProperty:\"weight\", writeProperty:\"pagerank\"})\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "lajr740m_Ixm",
        "outputId": "9ea0571f-e43a-415c-e1ee-1af4ac1ac506",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>writeMillis</th>\n",
              "      <th>nodePropertiesWritten</th>\n",
              "      <th>ranIterations</th>\n",
              "      <th>didConverge</th>\n",
              "      <th>centralityDistribution</th>\n",
              "      <th>postProcessingMillis</th>\n",
              "      <th>createMillis</th>\n",
              "      <th>computeMillis</th>\n",
              "      <th>configuration</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>12</td>\n",
              "      <td>33</td>\n",
              "      <td>20</td>\n",
              "      <td>False</td>\n",
              "      <td>{'p99': 3.6682729721069336, 'min': 0.149999618...</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>278</td>\n",
              "      <td>{'maxIterations': 20, 'writeConcurrency': 4, '...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   writeMillis  ...                                      configuration\n",
              "0           12  ...  {'maxIterations': 20, 'writeConcurrency': 4, '...\n",
              "\n",
              "[1 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 101
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Lastly, we execute the weighted Louvain algorithm to deduce the community structure and store the results in the database."
      ],
      "metadata": {
        "id": "nLLnxq9iqpcD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "read_query(\"\"\"\n",
        "CALL gds.louvain.write(\"matrix\", {relationshipWeightProperty:\"weight\", writeProperty:\"louvain\"})\n",
        "\"\"\")"
      ],
      "metadata": {
        "id": "QY9UMioU_hCI",
        "outputId": "c74349e7-61eb-4c64-e1af-5712cc6982f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 98
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>writeMillis</th>\n",
              "      <th>nodePropertiesWritten</th>\n",
              "      <th>modularity</th>\n",
              "      <th>modularities</th>\n",
              "      <th>ranLevels</th>\n",
              "      <th>communityCount</th>\n",
              "      <th>communityDistribution</th>\n",
              "      <th>postProcessingMillis</th>\n",
              "      <th>createMillis</th>\n",
              "      <th>computeMillis</th>\n",
              "      <th>configuration</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>13</td>\n",
              "      <td>33</td>\n",
              "      <td>0.08213</td>\n",
              "      <td>[0.07350871374383017, 0.08213048458616334]</td>\n",
              "      <td>2</td>\n",
              "      <td>16</td>\n",
              "      <td>{'p99': 11, 'min': 1, 'max': 11, 'mean': 2.062...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>893</td>\n",
              "      <td>{'maxIterations': 10, 'writeConcurrency': 4, '...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   writeMillis  ...                                      configuration\n",
              "0           13  ...  {'maxIterations': 10, 'writeConcurrency': 4, '...\n",
              "\n",
              "[1 rows x 11 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 102
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Open Neo4j Bloom to visualize the results"
      ],
      "metadata": {
        "id": "BWwukJRpqqsx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "pcDGw2p1_kcG"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}